package Parser;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import Client.DBTransaction;
import Structure.Word;

/**
 * This class parses all pages from the wiktionary dump (the xml file articles).
 * Furthermore it determine the word described at the page and split the content
 * of the page in its chapters. These list of chapters are given to an specific
 * implementation of IChapterInterpreter (specific to the language of the
 * wiktionary) to resolve the content and return all words on the page. At the
 * end. This class return all words with the there attributes given in the
 * wiktionary dump.
 * 
 * @author Tony
 * 
 */
public class PageParser {

	private final IChapterInterpreter chapterInterpreter;

	// statistic values

	public int skippedPages;
	public int notParsablePages;
	public int numberOfPages;
	private final int maxNumberOfPages = 0;

	private final String pPageBegin = "\\s*<page>\\s*";
	private final String pPageEnd = "\\s*.*?</page>\\s*";

	private final Pattern pWord = Pattern
			.compile("\\s*<title>([^:()]*?)</title>\\s*");
	private final Pattern pEveryTitle = Pattern
			.compile("\\s*<title>(.*?)</title>\\s*");

	/**
	 * Start regex for the not xml text 1 rest
	 */
	private final Pattern pTextBegin = Pattern
			.compile("\\s*<text xml:space=\"preserve\">(.*)");
	/**
	 * end regex for the not xml text
	 */
	private final String pTextEnd = ".*</text>";

	// to read the lines
	private BufferedReader bf;
	private String restOfLine = null;
	private String[] spezialChapter;

	// functions

	public PageParser(IChapterInterpreter pageInterpreter, String propertiesFile) {
		this.chapterInterpreter = pageInterpreter;

		// load the other regex from properties file
		Properties p = new Properties();
		try {
			BufferedReader bf = new BufferedReader(new FileReader(
					propertiesFile));

			p.load(bf);

			// load the special chapters if some exists
			String specials = p.getProperty("chapterName", "");
			if (specials.equals("")) {
				spezialChapter = new String[0];
			} else {
				spezialChapter = specials.replace("{", "\\{")
						.replace("}", "\\}").split(";");
			}

		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	public List<Word> parseSource(String source) {
		this.numberOfPages = 0;
		this.skippedPages = 0;
		this.notParsablePages = 0;

		LinkedList<Word> listWord = new LinkedList<Word>();
		try {
			// this.bf = new BufferedReader(new FileReader(new File(source)));
			this.bf = new BufferedReader(new InputStreamReader(
					new FileInputStream(new File(source)), "UTF-8"));

			String line;
			while ((line = this.nextLine()) != null) {

				if (line.matches(pPageBegin)) {
					numberOfPages++;

					// parse a page
					List<Word> newWords = parsePage();
					listWord.addAll(newWords);

					if (numberOfPages == maxNumberOfPages)
						break;
					if (numberOfPages % 10000 == 0)
						System.out.println(numberOfPages + " pages");
				} else
					skippedPages++;

			}

			bf.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}

		return listWord;
	}

	public List<Word> parsePage() throws IOException {
		String line;
		String wordName = null;

		List<Word> newWords = new LinkedList<Word>();

		while ((line = this.nextLine()) != null) {
			if (line.matches(pPageEnd)) {
				break;
			}

			// test if this line contains the title tag
			Matcher everyTitleMatcher = pEveryTitle.matcher(line);
			if (everyTitleMatcher.matches()) {

				// match the title to get the name of the word
				Matcher m = pWord.matcher(line);
				if (m.matches())
					wordName = m.group(1);
				else
					this.skippedPages++;
			}

			Matcher m = pTextBegin.matcher(line);
			if (wordName != null && m.matches()) {
				String rest = m.group(1);
				// set rest
				this.restOfLine = rest;

				// parse subtext to chapter
				List<Chapter> listChapter = parseSuptextToChapter(wordName);

				// interpret chapters and add new words to the list
				newWords.addAll(this.chapterInterpreter.parseChapters(wordName,
						listChapter));
			}

		}

		if (wordName != null && newWords.size() == 0)
			notParsablePages++;

		return newWords;
	}

	public List<Chapter> parseSuptextToChapter(String wordName)
			throws IOException {
		String line;

		boolean firstChapter = true;
		String chapterName = "";
		int chapterDepth = -1;

		StringBuilder chapterContent = new StringBuilder();
		LinkedList<Chapter> listChapter = new LinkedList<Chapter>();

		Pattern chapterPattern = Pattern.compile("\\s*(==[=]*).*(==[=]*)\\s*");

		while ((line = this.nextLine()) != null) {
			if (line.matches(pTextEnd)) {
				listChapter.add(new Chapter(chapterName, chapterContent
						.toString(), chapterDepth));
				break;
			}

			// if chapter (== ... ==)
			Matcher chapterMatcher = chapterPattern.matcher(line);
			if (chapterMatcher.matches()) {

				// add last chapter
				if (!firstChapter)
					listChapter.add(new Chapter(chapterName, chapterContent
							.toString(), chapterDepth));
				else
					firstChapter = false;

				// reset values for next chapter
				chapterName = line;
				chapterContent = new StringBuilder();
				chapterDepth = chapterMatcher.group(1).length() - 1;

				continue;

				// or if special chapter ( {{...}} )
			} else if (matchSpecialChapter(line)) {
				// add last chapter
				if (!firstChapter)
					listChapter.add(new Chapter(chapterName, chapterContent
							.toString(), chapterDepth));
				else
					firstChapter = false;

				// reset values for next chapter
				chapterName = line;
				chapterContent = new StringBuilder();

				continue;
			}

			chapterContent.append(line + "\n");

		}

		// interpret chapters and return the words
		return listChapter;
	}

	private boolean matchSpecialChapter(String line) {
		for (String regex : spezialChapter) {
			if (line.matches(regex))
				return true;
		}
		return false;
	}

	private String nextLine() throws IOException {

		// TODO possible point for a blacklist!

		if (restOfLine != null) {
			String puf = restOfLine;
			restOfLine = null;

			return puf;
		} else
			return bf.readLine();
	}

	public void addWordsToDB(List<Word> words) {
		DBTransaction t = new DBTransaction();
		t.connect();
		t.addWord(words);
	}

	public static void main(String[] args) {

		if (args.length < 2){
			System.out
					.println("please insert the path to the german pages" +
							" as first entry and the english as second");
			return;
		}
		
		

		// German parser
		PageParser gc = new PageParser(new GEChapterParser(
				"properties/DECrawler.properties"),
				"properties/DECrawler.properties");
		List<Word> words = gc
//				.parseSource("dewiktionary-20111124-pages-articles.xml");
				.parseSource(args[0]);
		gc.addWordsToDB(words);

		// English parser
		gc = new PageParser(new ENChapterParser(
				"properties/ENCrawler.properties"),
				"properties/ENCrawler.properties");
		words = gc.
//				parseSource("enwiktionary-20111121-pages-articles.xml");
				parseSource(args[1]);
		gc.addWordsToDB(words);

		// List<Word> words = gc.parseSource("example.xml");
		// List<Word> words = gc
		// .parseSource("dewiktionary-20111124-pages-articles.xml");

		// !!!!!!!!!!!!!! print the words to the console !!!!!!!!!!!!!!!!!!!!!
		// for (int i = 0; i < words.size(); i++)
		// System.out.println(words.get(i));

		// gc.writeWords(words);

		// !!!!!!!!!!!!!! add the words to the database !!!!!!!!!!!!!!!!!!!!!!
		// gc.addWordsToDB(words);

		// System.out.println("\nskipped:\t" + gc.skippedPages);
		// System.out.println("words:\t" + words.size());
		// System.out.println("number of pages:\t"+gc.numberOfPages);
		// System.out.println("number of skipped pages: "+gc.skippedPages);
		// System.out.println("number of not parsable pages: "+gc.notParsablePages);
	}

	public void writeWords(List<Word> words) {
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
					"words.txt")));
			for (int i = 0; i < words.size(); i++)
				bw.write(words.get(i) + "\n");

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

}
